package crawler.web.engine.custom;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import library.Tracer;
import library.functions.Function1PR;
import library.functions.Function2PR;
import library.wapper.IntWapper;

import org.apache.commons.lang.StringUtils;
import org.htmlparser.Node;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.tags.CompositeTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.springframework.stereotype.Component;

import crawler.web.component.HtmlDigger;

/**
 * @author waynechen Feb 20, 2012 12:04:46 AM
 */
@Component("NoiseDetector")
public class NoiseDetector {
	private final double RATIO = 22.2222;

	public boolean isNoise(final Node iNode) {
		NodeList children = iNode.getChildren();
		if (children == null || children.size() <= 0)
			return false;
		double links;
		double words;
		links = getNumLinks(iNode);
		words = getNumWords(iNode);

		double ratio = 0;
		if (words == 0)
			ratio = RATIO;
		else
			ratio = links / words;
		Tracer.trace("ratio", "噪音率: " + ratio);
		if (ratio > RATIO)
			return true;
		return false;
	}

	private Pattern shortPattern = Pattern.compile("^[\\/]*$");
	private Pattern jsCode = Pattern.compile(".*(function .*?\\(.*?\\).*?)+$", Pattern.DOTALL);

	public boolean isJsCode(String text) {
		Matcher m = jsCode.matcher(text);
		if (m.matches())
			return true;
		return false;
	}

	/**
	 * 判断一个串是不是无用的
	 * 
	 * @author waynechen
	 * @param text
	 * @return
	 */
	public boolean isNoise(String text) {
		if (StringUtils.isEmpty(text))
			return true;
		text = text.//
		replace("\n", "").//
		replace("\r", "").//
		replace("br /", "").//
		trim();
		if (text.length() <= 1)
			return true;
		else {
			Matcher m = shortPattern.matcher(text);
			if (m.matches())
				return true;
		}
		return false;
	}

	public double getTitleBodySimiarity(String title, String body) {
		int matchNum = 0;
		for (int i = 0; i < title.length(); i++) {
			if (body.indexOf(title.charAt(i)) >= 0) {
				++matchNum;
			}
		}
		double score = (double) matchNum / ((double) title.length());
		return score;
	}

	private int getNumWords(final Node iNode) {
		if (!(iNode instanceof CompositeTag))
			return 0;
		IntWapper $int = new IntWapper(0);
		HtmlDigger.travelTree((CompositeTag) iNode, $int, false, new Function2PR<Node, Object, Boolean>() {
			@Override
			public Boolean apply(Node a, Object b) {
				if (a instanceof TextNode) {
					TextNode aa = (TextNode) a;
					IntWapper $int = (IntWapper) b;
					$int.integer += getHTMLLen(aa.getText());
				}
				return null;
			}
		}, new Function1PR<Node, Boolean>() {
			@Override
			public Boolean apply(Node a) {
				if (a instanceof LinkTag)
					return false;
				return true;
			}
		});
		return $int.integer;
	}

	private int getHTMLLen(String text) {
		int len = 0;
		for (int i = 0; i < text.length(); i++) {
			if (text.charAt(i) == ' ') {

			} else if (text.charAt(i) == '　') {

			} else if (text.charAt(i) == '	') {

			} else if (text.charAt(i) == '&') {
				i += 5;
			} else
				++len;
		}
		if (len < 10)
			len = 0;
		return len;
	}

	private int getNumLinks(Node iNode) {
		if (!(iNode instanceof CompositeTag))
			return 0;
		IntWapper $int = new IntWapper(0);
		HtmlDigger.travelTree((CompositeTag) iNode, $int, false, new Function2PR<Node, Object, Boolean>() {
			@Override
			public Boolean apply(Node a, Object b) {
				if (a instanceof LinkTag) {
					IntWapper $int = (IntWapper) b;
					$int.integer++;
				}
				return null;
			}
		}, null);
		return $int.integer;
	}
}

//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
